package com.dmcb.trade.business.crawlers.author;

import com.dmcb.common.business.services.BaseService;
import com.dmcb.common.web.conversion.JsonResult;
import com.dmcb.common.business.utils.WebClientUtil;
import com.dmcb.trade.controller.ArticleCrawlerController;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

/**
 * Created by wangyong on 2017/6/7.
 */
@Service
public class ChaoQiCrawler extends BaseService {
    @Autowired
    private WebClientUtil webClientUtil;
    @Autowired
    private ArticleCrawlerController articleCrawlerController;
    org.slf4j.Logger logger = LoggerFactory.getLogger(ChaoQiCrawler.class);


    public JsonResult crawl() {
        int ero = 0;
        int success = 0;
        String link;
        int page = 1;
        WebClientUtil.wc = webClientUtil.initWc(false,false,true);
        while (true) {
            link = "http://www.chaoqi.net/dujiabaodao/index_" + page + ".html";
            Document doc = webClientUtil.getDoc(link);
            if (doc == null) {
                logger.info("解析失败，链接：" + link);
                break;
            }
            Elements elements = doc.select("ul#newsListUl>li");
            if (elements == null || elements.size() == 0) {
                break;
            }
            for (Element element : elements) {
                String src = element.select("a").attr("href");
                if (StringUtils.isBlank(link)) {
                    continue;
                }
                JsonResult jsonResult = articleCrawlerController.crawl(3897, src);
                if (jsonResult.getStatus() != JsonResult.STATUS_SUCCESS) {
                    if (jsonResult.getStatus() == JsonResult.STATUS_ERROR) {
                        ero++;
                    }
                    if (ero >= 10) {
                        return error("连续10次导入失败，请核实原因后再导入");
                    }
                    continue;
                }
                ero = 0;
                success++;
            }
            page++;
        }
        return success("成功添加" + success + "条文章");
    }

}
